{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Criteo CTR预估  ——  Wide and Deep\n",
    "\n",
    "    采用Wide and Deep模型，对Criteo提供的Kaggle竞赛数据进行CTR预估。该竞赛共包含11天的数据量，其中，10天为训练数据train，1天为测试数据test。 \n",
    "    \n",
    "    原始数据集包括： \n",
    "        1. train.csv：训练数据\n",
    "        2. eval.csv：测试数据\n",
    "    \n",
    "    \n",
    "## 一、导入相关包和数据\n",
    "\n",
    "\n",
    "### 1.1 Pandas读入\n",
    "\n",
    "    pandas是将整个数据集加载到内存中，再进行处理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据读取及基本处理\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clicked</th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>...</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2636</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>768</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>d4bb7bd8</td>\n",
       "      <td>2804effd</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>723b4dfd</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>b34f3128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>10003</td>\n",
       "      <td>22</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>8efede7f</td>\n",
       "      <td>7b06fafe</td>\n",
       "      <td>85684dc0</td>\n",
       "      <td>a458ea53</td>\n",
       "      <td>7ae4d78f</td>\n",
       "      <td>0</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>67a18c8c</td>\n",
       "      <td>2bf691b1</td>\n",
       "      <td>6aba8db0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1477</td>\n",
       "      <td>81</td>\n",
       "      <td>3</td>\n",
       "      <td>43</td>\n",
       "      <td>90</td>\n",
       "      <td>...</td>\n",
       "      <td>e5ba7672</td>\n",
       "      <td>281769c2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>73d06dde</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>aee52b6f</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>42851</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>e5ba7672</td>\n",
       "      <td>53515e19</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>5840adea</td>\n",
       "      <td>567ed6ad</td>\n",
       "      <td>0</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>6095f986</td>\n",
       "      <td>ea9a246c</td>\n",
       "      <td>03219b28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>623</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>8efede7f</td>\n",
       "      <td>88416823</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>423fab69</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   clicked  I1    I2  I3  I4     I5  I6  I7  I8  I9  ...        C17  \\\n",
       "0        0   1  2636   6   9    768  18   1  18  18  ...   d4bb7bd8   \n",
       "1        1   0     1  24   3  10003  22   6   4  21  ...   8efede7f   \n",
       "2        0   0     0   2   3   1477  81   3  43  90  ...   e5ba7672   \n",
       "3        1   0     1   3   0  42851  17   0   0  11  ...   e5ba7672   \n",
       "4        1   1    -1   0   0    623   0   5  12  12  ...   8efede7f   \n",
       "\n",
       "         C18        C19        C20        C21        C22        C23  \\\n",
       "0   2804effd          0          0   723b4dfd   ad3062eb   32c7478e   \n",
       "1   7b06fafe   85684dc0   a458ea53   7ae4d78f          0   32c7478e   \n",
       "2   281769c2          0          0   73d06dde   ad3062eb   3a171ecb   \n",
       "3   53515e19   21ddcdc9   5840adea   567ed6ad          0   32c7478e   \n",
       "4   88416823          0          0          0   ad3062eb   423fab69   \n",
       "\n",
       "         C24        C25        C26  \n",
       "0   b34f3128          0          0  \n",
       "1   67a18c8c   2bf691b1   6aba8db0  \n",
       "2   aee52b6f          0          0  \n",
       "3   6095f986   ea9a246c   03219b28  \n",
       "4          0          0          0  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 设置列名\n",
    "name = []\n",
    "for index in range(40) :\n",
    "    if index == 0 :\n",
    "        name.append(\"clicked\")\n",
    "    elif index > 0 and index <= 13 :\n",
    "        name.append(\"I\" + str(index))\n",
    "    else :\n",
    "        name.append(\"C\" + str(index-13))\n",
    "        \n",
    "        \n",
    "# 读取数据\n",
    "df_train = pd.read_csv(\"Criteo-Wide&Deep/data/train.csv\", names=name)\n",
    "df_eval = pd.read_csv(\"Criteo-Wide&Deep/data/eval.csv\", names=name)\n",
    "\n",
    "\n",
    "df_eval.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df_train\n",
    "del df_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " **字段说明：**  \n",
    " \n",
    "    clicked：    是否被点击\n",
    "    I1 - I13：     整数型特征，已进行脱敏处理\n",
    "    C1 - C26： 类别型特征，已进行Hash编码 \n",
    "     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 TensorFlow读入\n",
    "\n",
    "    将文件放入一个“读取器”中，一次读取一批文件。其中，Tensor张量设置为从其字符串名称到张量值映射的特征字典。字典结构是一对，第一个元素是映射列名称和特征值的DICT，第二个元素是标签值。回想一下，张量只是一个n维数组的通称。\n",
    "    如， { \n",
    "              'age':                [ 39, 50, 38, 53, 28, … ], \n",
    "              'marital_status':  [ 'Married-civ-spouse', 'Never-married', 'Widowed', 'Widowed' … ],\n",
    "               ...\n",
    "              'gender':            ['Male', 'Female', 'Male', 'Male', 'Female',, … ], \n",
    "            } , \n",
    "            [ 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using TensorFlow version 1.13.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 数据读取及基本处理\n",
    "from __future__ import absolute_import\n",
    "\n",
    "import time\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "\n",
    "\n",
    "print(\"Using TensorFlow version %s\\n\" % (tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature columns are:  ['I1', 'I2', 'I3', 'I4', 'I5', 'I6', 'I7', 'I8', 'I9', 'I10', 'I11', 'I12', 'I13', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21', 'C22', 'C23', 'C24', 'C25', 'C26'] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 设置列名\n",
    "CONTINUOUS_COLUMNS =  [\"I\"+str(i) for i in range(1,14)]\n",
    "CATEGORICAL_COLUMNS = [\"C\"+str(i) for i in range(1,27)]\n",
    "LABEL_COLUMN = [\"clicked\"]\n",
    "\n",
    "TRAIN_DATA_COLUMNS = LABEL_COLUMN + CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n",
    "FEATURE_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n",
    "print('Feature columns are: ', FEATURE_COLUMNS, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input function configured\n"
     ]
    }
   ],
   "source": [
    "# 读入数据\n",
    "BATCH_SIZE = 400\n",
    "\n",
    "def generate_input_fn(filename, batch_size=BATCH_SIZE):\n",
    "    def _input_fn():\n",
    "        # 读取批量大小行数\n",
    "        filename_queue = tf.train.string_input_producer([filename])\n",
    "        reader = tf.TextLineReader()\n",
    "        key, value = reader.read_up_to(filename_queue, num_records=batch_size)\n",
    "        \n",
    "        # 初始化数据格式：1 int label, 13 ints, 26 strings\n",
    "        cont_defaults = [ [0] for i in range(1,14) ]\n",
    "        cate_defaults = [ [\" \"] for i in range(1,27) ]\n",
    "        label_defaults = [ [0] ]\n",
    "        column_headers = TRAIN_DATA_COLUMNS\n",
    "        # 标签是数据的第一列，且记录默认值\n",
    "        record_defaults = label_defaults + cont_defaults + cate_defaults\n",
    "\n",
    "        # 解码刚刚读出的csv数据，由于不会直接返回字典，故使用zip \n",
    "        columns = tf.decode_csv(value, record_defaults=record_defaults)\n",
    "        all_columns = dict(zip(column_headers, columns))\n",
    "        \n",
    "        # 用dict.pop() 拿到标签值\n",
    "        labels = all_columns.pop(LABEL_COLUMN[0])\n",
    "        \n",
    "        # 剩下的是特征值列\n",
    "        features = all_columns \n",
    "\n",
    "        # 处理稀疏的类别型特征，增加最后一维（why）\n",
    "        for feature_name in CATEGORICAL_COLUMNS:\n",
    "            features[feature_name] = tf.expand_dims(features[feature_name], -1)\n",
    "\n",
    "        return features, labels\n",
    "\n",
    "    return _input_fn\n",
    "\n",
    "print('input function configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二、特征工程\n",
    "\n",
    "\n",
    "### 2.1 稀疏列值\n",
    "\n",
    "    sparse_column_with_keys()：根据key构建低维离散特征\n",
    "    sparse_column_with_hash_bucket()：自动映射值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "Wide/Sparse columns configured\n"
     ]
    }
   ],
   "source": [
    "# 方法一：\n",
    "# C1 = tf.contrib.layers.sparse_column_with_hash_bucket('C1', hash_bucket_size=1000)\n",
    "# C2 = tf.contrib.layers.sparse_column_with_hash_bucket('C2', hash_bucket_size=1000)\n",
    "# C3 = tf.contrib.layers.sparse_column_with_hash_bucket('C3', hash_bucket_size=1000)\n",
    "# ...\n",
    "# Cn = tf.contrib.layers.sparse_column_with_hash_bucket('Cn', hash_bucket_size=1000)\n",
    "# wide_columns = [C1, C2, C3, ... , Cn]\n",
    "\n",
    "\n",
    "# 方法二：\n",
    "wide_columns = []\n",
    "for name in CATEGORICAL_COLUMNS:\n",
    "    wide_columns.append(tf.contrib.layers.sparse_column_with_hash_bucket(\n",
    "            name, hash_bucket_size=1000))\n",
    "\n",
    "print('Wide/Sparse columns configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 连续列值\n",
    "    \n",
    "    real_valued_column()：构建连续型实数特征\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deep/continuous columns configured\n"
     ]
    }
   ],
   "source": [
    "# 方法一：\n",
    "# I1 = tf.contrib.layers.real_valued_column(\"I1\")\n",
    "# I2 = tf.contrib.layers.real_valued_column(\"I2\")\n",
    "# I3 = tf.contrib.layers.real_valued_column(\"I3\")\n",
    "# ...\n",
    "# In = tf.contrib.layers.real_valued_column(\"In\")\n",
    "# deep_columns = [I1, I2, I3, ... , In]\n",
    "\n",
    "\n",
    "# 方法二：\n",
    "deep_columns = []\n",
    "for name in CONTINUOUS_COLUMNS:\n",
    "    deep_columns.append(tf.contrib.layers.real_valued_column(name))\n",
    "\n",
    "print('deep/continuous columns configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 特征组合\n",
    "\n",
    "    用于Wide Model，包括已有的稀疏类别型特征列sparse categorical columns，以及后续的hashed、bucket和feature crossed columns。\n",
    "    \n",
    "    1. bucketizing：特征分段，将连续型特征转换为类别型特征\n",
    "    2. feature crossing：对意义相近的特征进行组合，也就是特征交叉，如教育程度和职业。注意，但只有类别型特征可以进行交叉。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformations complete\n"
     ]
    }
   ],
   "source": [
    "# 举例：年龄分段、特征二阶组合、三阶组合等\n",
    "# age_buckets = tf.contrib.layers.bucketized_column(age,\n",
    "#             boundaries=[ 18, 25, 30, 35, 40, 45, 50, 55, 60, 65 ])\n",
    "# education_occupation = tf.contrib.layers.crossed_column([education, occupation], \n",
    "#                                                         hash_bucket_size=int(1e4))\n",
    "# age_race_occupation = tf.contrib.layers.crossed_column([age_buckets, race, occupation], \n",
    "#                                                        hash_bucket_size=int(1e6))\n",
    "# country_occupation = tf.contrib.layers.crossed_column([native_country, occupation], \n",
    "#                                                       hash_bucket_size=int(1e4))\n",
    "\n",
    "print('Transformations complete')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 特征列嵌入（Column embeddings）\n",
    "\n",
    "    用于Deep Model，包括嵌入式的类别列（the composed of embedded categorical columns）和连续型的实数列（the continuous real-valued columns），主要是将稀疏的类别型张量通过embedding转变为低维稠密的实数向量，可以先从简单的8维嵌入开始。\n",
    "    \n",
    "参考资料：[Vector Representations Words](https://www.tensorflow.org/tutorials/word2vec/)、\n",
    "               [Word Embedding](https://en.wikipedia.org/wiki/Word_embedding)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "WARNING:tensorflow:The default stddev value of initializer was changed from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" in core implementation (tf.feature_column.embedding_column).\n",
      "wide and deep columns configured\n"
     ]
    }
   ],
   "source": [
    "# wide列和deep列，例如：\n",
    "\n",
    "# wide_columns = [gender, race, native_country,\n",
    "#       education, occupation, workclass,\n",
    "#       marital_status, relationship,\n",
    "#\n",
    "#       age_buckets, education_occupation,\n",
    "#       age_race_occupation, country_occupation]\n",
    "\n",
    "# deep_columns = [\n",
    "#   tf.contrib.layers.embedding_column(workclass, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(education, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(marital_status, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(gender, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(relationship, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(race, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(native_country, dimension=8),\n",
    "#   tf.contrib.layers.embedding_column(occupation, dimension=8),\n",
    "#   age,\n",
    "#   education_num,\n",
    "#   capital_gain,\n",
    "#   capital_loss,\n",
    "#   hours_per_week,\n",
    "# ]\n",
    "\n",
    "\n",
    "\n",
    "# embedding类别型列加入到deep columns中，原本深列只有连续型\n",
    "for col in wide_columns:\n",
    "    deep_columns.append(tf.contrib.layers.embedding_column(col, \n",
    "                                                           dimension=8))\n",
    "\n",
    "print('wide and deep columns configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 三、构建模型\n",
    "\n",
    "* **Wide**：线性分类器（Linear Classifier）\n",
    "* **Deep**：深度神经网络分类器（Deep Neural Net Classifier）\n",
    "* **Wide & Deep**：组合的线性和深度神经网络分类器（Combined Linear and Deep Classifier）\n",
    "\n",
    "    其中，wide部分主要是高维特征+特征组合的LR模型，其宽线性部分可以利用交叉特征去有效地记忆稀疏特征之间的相互作用。而deep的部分则主要由embedding层、隐层和softmax层组成，其深层神经网络可以通过挖掘特征之间的相互作用，提升模型之间的泛化能力。\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_dir(model_type):\n",
    "    # 返回创建模型信息，如：models/model_WIDE_AND_DEEP_1493043407\n",
    "    return 'models/model_' + model_type + '_' + str(int(time.time()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建模型及其存储路径 \n",
    "def get_model(model_type, model_dir):\n",
    "    print(\"Model directory = %s\" % model_dir)\n",
    "    \n",
    "    # 模型每100step保存一次参数，但在一个有更多样本的真实系统中，还是可能选择较少地保存检查点\n",
    "    runconfig = tf.contrib.learn.RunConfig(\n",
    "        save_checkpoints_secs = None,\n",
    "        save_checkpoints_steps = 100,\n",
    "    )\n",
    "    \n",
    "    m = None  # 待创建的模型\n",
    "    \n",
    "    # Linear Classifier\n",
    "    if model_type == 'WIDE':\n",
    "        m = tf.contrib.learn.LinearClassifier(\n",
    "            model_dir = model_dir, \n",
    "            feature_columns = wide_columns)\n",
    "\n",
    "    # Deep Neural Net Classifier\n",
    "    if model_type == 'DEEP':\n",
    "        m = tf.contrib.learn.DNNClassifier(\n",
    "            model_dir = model_dir,\n",
    "            feature_columns = deep_columns,\n",
    "            hidden_units = [100, 50, 25])  # hidden_units/dnn_hidden_units用于指定网络深部各层的大小\n",
    "    \n",
    "    # Combined Linear and Deep Classifier\n",
    "    if model_type == 'WIDE_AND_DEEP':\n",
    "        m = tf.contrib.learn.DNNLinearCombinedClassifier(\n",
    "            model_dir = model_dir,\n",
    "            linear_feature_columns = wide_columns,\n",
    "            dnn_feature_columns = deep_columns,\n",
    "            dnn_hidden_units = [100, 50, 25],\n",
    "            config = runconfig)\n",
    "        \n",
    "    print('estimator built')\n",
    "    \n",
    "    return m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Wide 模型\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model directory = models/model_WIDE_1571758061\n",
      "WARNING:tensorflow:From <ipython-input-12-26bedce88d1c>:8: RunConfig.__init__ (from tensorflow.contrib.learn.python.learn.estimators.run_config) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.RunConfig instead.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/linear.py:469: multi_class_head (from tensorflow.contrib.learn.python.learn.estimators.head) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.contrib.estimator.*_head.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1179: BaseEstimator.__init__ (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please replace uses of any Estimator from tf.contrib.learn with an Estimator from tf.estimator.*\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x136ab7f98>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_session_config': None, '_save_checkpoints_steps': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_WIDE_1571758061'}\n",
      "estimator built\n"
     ]
    }
   ],
   "source": [
    "MODEL_TYPE = 'WIDE'\n",
    "wide_model_dir = create_model_dir(model_type=MODEL_TYPE)\n",
    "wide_model = get_model(model_type=MODEL_TYPE, model_dir=wide_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Canned estimators可以一行代码就创建深度模型，我们再返回一个Evaluable实例\n",
    "from tensorflow.contrib.learn.python.learn import evaluable\n",
    "isinstance(wide_model, evaluable.Evaluable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Deep 模型\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model directory = models/model_DEEP_1571758068\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x136abf3c8>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_session_config': None, '_save_checkpoints_steps': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_DEEP_1571758068'}\n",
      "estimator built\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_TYPE = 'DEEP'\n",
    "deep_model_dir = create_model_dir(model_type=MODEL_TYPE)\n",
    "deep_model = get_model(model_type=MODEL_TYPE, model_dir=deep_model_dir)\n",
    "\n",
    "isinstance(deep_model, evaluable.Evaluable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Wide&Deep 模型\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model directory = models/model_WIDE_AND_DEEP_1571758070\n",
      "WARNING:tensorflow:From <ipython-input-12-26bedce88d1c>:33: calling DNNLinearCombinedClassifier.__init__ (from tensorflow.contrib.learn.python.learn.estimators.dnn_linear_combined) with fix_global_step_increment_bug=False is deprecated and will be removed after 2017-04-15.\n",
      "Instructions for updating:\n",
      "Please set fix_global_step_increment_bug=True and update training steps in your pipeline. See pydoc for details.\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x136abf400>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_protocol': None, '_session_config': None, '_save_checkpoints_steps': 100, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_WIDE_AND_DEEP_1571758070'}\n",
      "estimator built\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_TYPE = 'WIDE_AND_DEEP'\n",
    "model_dir = create_model_dir(model_type=MODEL_TYPE)\n",
    "m = get_model(model_type=MODEL_TYPE, model_dir=model_dir)\n",
    "\n",
    "isinstance(m, evaluable.Evaluable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 四、训练模型\n",
    "\n",
    "### 4.1 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CLOUD：云端数据\n",
    "#train_file = \"gs://dataset-uploader/criteo-kaggle/medium_version/train.csv\"\n",
    "#eval_file  = \"gs://dataset-uploader/criteo-kaggle/medium_version/eval.csv\"\n",
    "\n",
    "# LOCAL：本地数据\n",
    "train_file = \"Criteo-Wide&Deep/data/train.csv\"\n",
    "eval_file  = \"Criteo-Wide&Deep/data/eval.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 训练wide模型\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-6-ea22554dcd91>:7: string_input_producer (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.from_tensor_slices(string_tensor).shuffle(tf.shape(input_tensor, out_type=tf.int64)[0]).repeat(num_epochs)`. If `shuffle=False`, omit the `.shuffle(...)`.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py:278: input_producer (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.from_tensor_slices(input_tensor).shuffle(tf.shape(input_tensor, out_type=tf.int64)[0]).repeat(num_epochs)`. If `shuffle=False`, omit the `.shuffle(...)`.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py:190: limit_epochs (from tensorflow.python.training.input) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.Dataset.from_tensors(tensor).repeat(num_epochs)`.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py:199: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py:199: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py:202: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From <ipython-input-6-ea22554dcd91>:8: TextLineReader.__init__ (from tensorflow.python.ops.io_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Queue-based input pipelines have been replaced by `tf.data`. Use `tf.data.TextLineDataset`.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/metrics_impl.py:788: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:677: ModelFnOps.__new__ (from tensorflow.contrib.learn.python.learn.estimators.model_fn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.EstimatorSpec. You can use the `estimator_spec` method to create an equivalent one.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/monitored_session.py:809: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into models/model_WIDE_1571758061/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.6931474, step = 1\n",
      "INFO:tensorflow:global_step/sec: 54.816\n",
      "INFO:tensorflow:loss = 0.45659465, step = 101 (1.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.014\n",
      "INFO:tensorflow:loss = 0.53344035, step = 201 (0.730 sec)\n",
      "INFO:tensorflow:global_step/sec: 136.982\n",
      "INFO:tensorflow:loss = 0.47971246, step = 301 (0.730 sec)\n",
      "INFO:tensorflow:global_step/sec: 136.505\n",
      "INFO:tensorflow:loss = 0.5206535, step = 401 (0.733 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.81\n",
      "INFO:tensorflow:loss = 0.5690068, step = 501 (0.726 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.924\n",
      "INFO:tensorflow:loss = 0.4670084, step = 601 (0.725 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.262\n",
      "INFO:tensorflow:loss = 0.5033304, step = 701 (0.729 sec)\n",
      "INFO:tensorflow:global_step/sec: 136.775\n",
      "INFO:tensorflow:loss = 0.56871617, step = 801 (0.731 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.882\n",
      "INFO:tensorflow:loss = 0.4968151, step = 901 (0.725 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.774\n",
      "INFO:tensorflow:loss = 0.49897653, step = 1001 (0.726 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.139\n",
      "INFO:tensorflow:loss = 0.57331276, step = 1101 (0.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 125.456\n",
      "INFO:tensorflow:loss = 0.5235764, step = 1201 (0.797 sec)\n",
      "INFO:tensorflow:global_step/sec: 133.897\n",
      "INFO:tensorflow:loss = 0.46780455, step = 1301 (0.747 sec)\n",
      "INFO:tensorflow:global_step/sec: 135.552\n",
      "INFO:tensorflow:loss = 0.44247612, step = 1401 (0.738 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.446\n",
      "INFO:tensorflow:loss = 0.5041065, step = 1501 (0.727 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.376\n",
      "INFO:tensorflow:loss = 0.47314006, step = 1601 (0.728 sec)\n",
      "INFO:tensorflow:global_step/sec: 136.438\n",
      "INFO:tensorflow:loss = 0.5302299, step = 1701 (0.733 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.318\n",
      "INFO:tensorflow:loss = 0.54489845, step = 1801 (0.728 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.045\n",
      "INFO:tensorflow:loss = 0.49571082, step = 1901 (0.901 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into models/model_WIDE_1571758061/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.5170178.\n",
      "wide model fit done\n",
      "CPU times: user 43.1 s, sys: 3.26 s, total: 46.4 s\n",
      "Wall time: 26.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_sample_size = 800000\n",
    "train_steps = train_sample_size/BATCH_SIZE # 800000/400 = 2000\n",
    "\n",
    "wide_model.fit(input_fn=generate_input_fn(train_file, BATCH_SIZE), steps=train_steps)\n",
    "print('wide model fit done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "INFO:tensorflow:Saving checkpoints for 2000 into models/model_WIDE_1571737141/model.ckpt.  \n",
    "INFO:tensorflow:Loss for final step: 0.5170178.  \n",
    "wide model fit done  \n",
    "CPU times: user 42.9 s, sys: 3.27 s, total: 46.2 s  \n",
    "Wall time: 26.4 s  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 训练deep模型\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into models/model_DEEP_1571758068/model.ckpt.\n",
      "INFO:tensorflow:loss = 141.48999, step = 1\n",
      "INFO:tensorflow:global_step/sec: 40.0736\n",
      "INFO:tensorflow:loss = 8.78139, step = 101 (2.497 sec)\n",
      "INFO:tensorflow:global_step/sec: 80.2892\n",
      "INFO:tensorflow:loss = 0.64537656, step = 201 (1.245 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.9909\n",
      "INFO:tensorflow:loss = 0.75217813, step = 301 (1.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.4706\n",
      "INFO:tensorflow:loss = 1.207735, step = 401 (1.212 sec)\n",
      "INFO:tensorflow:global_step/sec: 80.4943\n",
      "INFO:tensorflow:loss = 0.63722014, step = 501 (1.242 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.6787\n",
      "INFO:tensorflow:loss = 0.5848158, step = 601 (1.195 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.9154\n",
      "INFO:tensorflow:loss = 0.54719526, step = 701 (1.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.1761\n",
      "INFO:tensorflow:loss = 0.587851, step = 801 (1.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.6416\n",
      "INFO:tensorflow:loss = 0.5452785, step = 901 (1.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.9906\n",
      "INFO:tensorflow:loss = 0.5455085, step = 1001 (1.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.9956\n",
      "INFO:tensorflow:loss = 0.6262706, step = 1101 (1.191 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.4898\n",
      "INFO:tensorflow:loss = 0.6141891, step = 1201 (1.227 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.9775\n",
      "INFO:tensorflow:loss = 0.52758604, step = 1301 (1.221 sec)\n",
      "INFO:tensorflow:global_step/sec: 59.2842\n",
      "INFO:tensorflow:loss = 0.5398122, step = 1401 (1.686 sec)\n",
      "INFO:tensorflow:global_step/sec: 74.9651\n",
      "INFO:tensorflow:loss = 0.57200956, step = 1501 (1.334 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.6166\n",
      "INFO:tensorflow:loss = 0.48721147, step = 1601 (1.196 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.6735\n",
      "INFO:tensorflow:loss = 0.54317254, step = 1701 (1.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.4889\n",
      "INFO:tensorflow:loss = 0.58396435, step = 1801 (1.212 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.9341\n",
      "INFO:tensorflow:loss = 0.5228317, step = 1901 (1.284 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into models/model_DEEP_1571758068/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.5537065.\n",
      "deep model fit done\n",
      "CPU times: user 1min 10s, sys: 4.52 s, total: 1min 15s\n",
      "Wall time: 36.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_sample_size = 800000\n",
    "train_steps = train_sample_size/BATCH_SIZE # 800000/400 = 2000\n",
    "\n",
    "deep_model.fit(input_fn=generate_input_fn(train_file, BATCH_SIZE), steps=train_steps)\n",
    "print('deep model fit done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "INFO:tensorflow:Saving checkpoints for 2000 into models/model_DEEP_1571737154/model.ckpt.  \n",
    "INFO:tensorflow:Loss for final step: 0.57385653.  \n",
    "deep model fit done  \n",
    "CPU times: user 1min 12s, sys: 4.86 s, total: 1min 17s  \n",
    "Wall time: 39.2 s  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.4 训练wide and deep模型\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:loss = 111.5408, step = 2\n",
      "INFO:tensorflow:Saving checkpoints for 102 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.8407\n",
      "INFO:tensorflow:loss = 2.9478478, step = 202 (9.720 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 204 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.4313\n",
      "INFO:tensorflow:Saving checkpoints for 306 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 55.496\n",
      "INFO:tensorflow:loss = 0.53595835, step = 402 (3.874 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 408 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 66.5111\n",
      "INFO:tensorflow:Saving checkpoints for 510 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "INFO:tensorflow:global_step/sec: 68.0637\n",
      "INFO:tensorflow:loss = 0.48610228, step = 602 (2.981 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 612 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 66.6499\n",
      "INFO:tensorflow:Saving checkpoints for 714 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.884\n",
      "INFO:tensorflow:loss = 0.5227471, step = 802 (2.995 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 816 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 66.7585\n",
      "INFO:tensorflow:Saving checkpoints for 918 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 68.0442\n",
      "INFO:tensorflow:loss = 0.56403, step = 1002 (2.988 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1020 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.5601\n",
      "INFO:tensorflow:Saving checkpoints for 1122 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.9147\n",
      "INFO:tensorflow:loss = 0.4653895, step = 1202 (2.989 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1224 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.0344\n",
      "INFO:tensorflow:Saving checkpoints for 1326 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 65.5348\n",
      "INFO:tensorflow:loss = 0.5072491, step = 1402 (3.052 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1428 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 66.806\n",
      "INFO:tensorflow:Saving checkpoints for 1530 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.48\n",
      "INFO:tensorflow:loss = 0.5651559, step = 1602 (2.999 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1632 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.2993\n",
      "INFO:tensorflow:Saving checkpoints for 1734 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.5731\n",
      "INFO:tensorflow:loss = 0.4925522, step = 1802 (2.994 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1836 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.8431\n",
      "INFO:tensorflow:Saving checkpoints for 1938 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 68.6962\n",
      "INFO:tensorflow:loss = 0.5006567, step = 2002 (3.127 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2002 into models/model_WIDE_AND_DEEP_1571758070/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.5006567.\n",
      "wide and deep model fit done\n",
      "CPU times: user 1min 20s, sys: 5.2 s, total: 1min 25s\n",
      "Wall time: 56 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_sample_size = 800000\n",
    "train_steps = train_sample_size/BATCH_SIZE # 800000/400 = 2000\n",
    "\n",
    "m.fit(input_fn=generate_input_fn(train_file, BATCH_SIZE), steps=train_steps)\n",
    "print('wide and deep model fit done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "INFO:tensorflow:Saving checkpoints for 2002 into models/model_WIDE_AND_DEEP_1571737173/model.ckpt.  \n",
    "INFO:tensorflow:Loss for final step: 0.4946544.  \n",
    "wide and deep model fit done  \n",
    "CPU times: user 1min 25s, sys: 5.68 s, total: 1min 31s  \n",
    "Wall time: 1min 3s  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**此时训练集上的对比如下：**\n",
    "\n",
    "模型 | Loss值 |  时间  \n",
    "-|-|-\n",
    "Wide | 0.5170178 | 26.4 s |\n",
    "Deep | 0.57385653 | 39.2 s |\n",
    "Wide&Deep | 0.4946544 | 1min 3s |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 五、评估模型\n",
    "\n",
    "### 5.1 评估wide模型\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:31:20Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_1571758061/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [50/500]\n",
      "INFO:tensorflow:Evaluation [100/500]\n",
      "INFO:tensorflow:Evaluation [150/500]\n",
      "INFO:tensorflow:Evaluation [200/500]\n",
      "INFO:tensorflow:Evaluation [250/500]\n",
      "INFO:tensorflow:Evaluation [300/500]\n",
      "INFO:tensorflow:Evaluation [350/500]\n",
      "INFO:tensorflow:Evaluation [400/500]\n",
      "INFO:tensorflow:Evaluation [450/500]\n",
      "INFO:tensorflow:Evaluation [500/500]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:31:25\n",
      "INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.766125, accuracy/baseline_label_mean = 0.251165, accuracy/threshold_0.500000_mean = 0.766125, auc = 0.72278744, auc_precision_recall = 0.47572494, global_step = 2000, labels/actual_label_mean = 0.251165, labels/prediction_mean = 0.26990917, loss = 0.50244325, precision/positive_threshold_0.500000_mean = 0.6087969, recall/positive_threshold_0.500000_mean = 0.19260247\n",
      "wide model evaluate done\n",
      "wide model Accuracy: 0.766125\n",
      "{'loss': 0.50244325, 'accuracy': 0.766125, 'labels/prediction_mean': 0.26990917, 'labels/actual_label_mean': 0.251165, 'accuracy/baseline_label_mean': 0.251165, 'auc': 0.72278744, 'auc_precision_recall': 0.47572494, 'accuracy/threshold_0.500000_mean': 0.766125, 'precision/positive_threshold_0.500000_mean': 0.6087969, 'recall/positive_threshold_0.500000_mean': 0.19260247, 'global_step': 2000}\n",
      "CPU times: user 11.1 s, sys: 772 ms, total: 11.8 s\n",
      "Wall time: 7.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "eval_sample_size = 200000\n",
    "eval_steps = eval_sample_size/BATCH_SIZE # 200000/400 = 500\n",
    "\n",
    "results = wide_model.evaluate(input_fn=generate_input_fn(eval_file), \n",
    "                     steps=eval_steps)\n",
    "print('wide model evaluate done')\n",
    "\n",
    "print('wide model Accuracy: %s' % results['accuracy'])\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "wide model evaluate done  \n",
    "wide model Accuracy: 0.766125  \n",
    "CPU times: user 11.1 s, sys: 844 ms, total: 12 s  \n",
    "Wall time: 7.77 s  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2 评估deep模型\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:31:38Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_DEEP_1571758068/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [50/500]\n",
      "INFO:tensorflow:Evaluation [100/500]\n",
      "INFO:tensorflow:Evaluation [150/500]\n",
      "INFO:tensorflow:Evaluation [200/500]\n",
      "INFO:tensorflow:Evaluation [250/500]\n",
      "INFO:tensorflow:Evaluation [300/500]\n",
      "INFO:tensorflow:Evaluation [350/500]\n",
      "INFO:tensorflow:Evaluation [400/500]\n",
      "INFO:tensorflow:Evaluation [450/500]\n",
      "INFO:tensorflow:Evaluation [500/500]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:31:45\n",
      "INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.752145, accuracy/baseline_label_mean = 0.251165, accuracy/threshold_0.500000_mean = 0.752145, auc = 0.67770857, auc_precision_recall = 0.4105076, global_step = 2000, labels/actual_label_mean = 0.251165, labels/prediction_mean = 0.25075603, loss = 0.53638744, precision/positive_threshold_0.500000_mean = 0.5633614, recall/positive_threshold_0.500000_mean = 0.058586985\n",
      "deep model evaluate done\n",
      "deep model Accuracy: 0.752145\n",
      "{'loss': 0.53638744, 'accuracy': 0.752145, 'labels/prediction_mean': 0.25075603, 'labels/actual_label_mean': 0.251165, 'accuracy/baseline_label_mean': 0.251165, 'auc': 0.67770857, 'auc_precision_recall': 0.4105076, 'accuracy/threshold_0.500000_mean': 0.752145, 'precision/positive_threshold_0.500000_mean': 0.5633614, 'recall/positive_threshold_0.500000_mean': 0.058586985, 'global_step': 2000}\n",
      "CPU times: user 16 s, sys: 995 ms, total: 17 s\n",
      "Wall time: 9.75 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "eval_sample_size = 200000\n",
    "eval_steps = eval_sample_size/BATCH_SIZE # 200000/400 = 500\n",
    "\n",
    "results = deep_model.evaluate(input_fn=generate_input_fn(eval_file), \n",
    "                     steps=eval_steps)\n",
    "print('deep model evaluate done')\n",
    "\n",
    "print('deep model Accuracy: %s' % results['accuracy'])\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "deep model evaluate done  \n",
    "deep model Accuracy: 0.75337  \n",
    "CPU times: user 16.7 s, sys: 1.29 s, total: 18 s  \n",
    "Wall time: 11.3 s  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.3 评估wide and deep模型\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:31:59Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758070/model.ckpt-2002\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [50/500]\n",
      "INFO:tensorflow:Evaluation [100/500]\n",
      "INFO:tensorflow:Evaluation [150/500]\n",
      "INFO:tensorflow:Evaluation [200/500]\n",
      "INFO:tensorflow:Evaluation [250/500]\n",
      "INFO:tensorflow:Evaluation [300/500]\n",
      "INFO:tensorflow:Evaluation [350/500]\n",
      "INFO:tensorflow:Evaluation [400/500]\n",
      "INFO:tensorflow:Evaluation [450/500]\n",
      "INFO:tensorflow:Evaluation [500/500]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:32:08\n",
      "INFO:tensorflow:Saving dict for global step 2002: accuracy = 0.765315, accuracy/baseline_label_mean = 0.251165, accuracy/threshold_0.500000_mean = 0.765315, auc = 0.72085005, auc_precision_recall = 0.4697164, global_step = 2002, labels/actual_label_mean = 0.251165, labels/prediction_mean = 0.25295934, loss = 0.5040875, precision/positive_threshold_0.500000_mean = 0.6319667, recall/positive_threshold_0.500000_mean = 0.15710787\n",
      "wide and deep model evaluate done\n",
      "wide and deep model Accuracy: 0.765315\n",
      "{'loss': 0.5040875, 'accuracy': 0.765315, 'labels/prediction_mean': 0.25295934, 'labels/actual_label_mean': 0.251165, 'accuracy/baseline_label_mean': 0.251165, 'auc': 0.72085005, 'auc_precision_recall': 0.4697164, 'accuracy/threshold_0.500000_mean': 0.765315, 'precision/positive_threshold_0.500000_mean': 0.6319667, 'recall/positive_threshold_0.500000_mean': 0.15710787, 'global_step': 2002}\n",
      "CPU times: user 21.4 s, sys: 1.29 s, total: 22.7 s\n",
      "Wall time: 13.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "eval_sample_size = 200000\n",
    "eval_steps = eval_sample_size/BATCH_SIZE # 200000/400 = 500\n",
    "\n",
    "results = m.evaluate(input_fn=generate_input_fn(eval_file), \n",
    "                     steps=eval_steps)\n",
    "print('wide and deep model evaluate done')\n",
    "\n",
    "print('wide and deep model Accuracy: %s' % results['accuracy'])\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "wide and deep model evaluate done  \n",
    "wide and deep model Accuracy: 0.76597  \n",
    "CPU times: user 22.5 s, sys: 1.44 s, total: 24 s  \n",
    "Wall time: 15.3 s  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**此时校验集上的对比如下：**\n",
    "\n",
    "模型 | Accuracy值 |  时间  \n",
    "-|-|-\n",
    "Wide | 0.766125 | 7.77 s |\n",
    "Deep | 0.75337 | 11.3 s |\n",
    "Wide&Deep | 0.76597 | 15.3 s |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型预测\n",
    "# def pred_fn():\n",
    "#     sample = [ 0, 127, 1, 3, 1683, 19, 26, 17, 475, 0, 9, 0, 3, \"05db9164\", \"8947f767\", \"11c9d79e\", \"52a787c8\", \"4cf72387\", \"fbad5c96\", \"18671b18\", \"0b153874\", \"a73ee510\", \"ceb10289\", \"77212bd7\", \"79507c6b\", \"7203f04e\", \"07d13a8f\", \"2c14c412\", \"49013ffe\", \"8efede7f\", \"bd17c3da\", \"f6a3e43b\", \"a458ea53\", \"35cd95c9\", \"ad3062eb\", \"c7dc6720\", \"3fdb382b\", \"010f6491\", \"49d68486\"]\n",
    "#     sample_dict = dict(zip(FEATURE_COLUMNS, sample))\n",
    "    \n",
    "#     for feature_name in CATEGORICAL_COLUMNS:\n",
    "#         sample_dict[feature_name] = tf.expand_dims(sample_dict[feature_name], -1)\n",
    "        \n",
    "#     for feature_name in CONTINUOUS_COLUMNS:\n",
    "#         sample_dict[feature_name] = tf.constant(sample_dict[feature_name], dtype=tf.int32)\n",
    "#     print(sample_dict)\n",
    "\n",
    "#     return sample_dict\n",
    "\n",
    "# m.predict(input_fn=pred_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 六、保存模型\n",
    "\n",
    "### 6.1 导出模型\n",
    "\n",
    "将训练好的模型通过`export_savedmodel()`函数导出为`saved_model.pb`文件和`variables` 文件夹，其主要是存储了训练过的权重参数。\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.contrib.learn.python.learn.utils import input_fn_utils\n",
    "\n",
    "def column_to_dtype(column):\n",
    "    if column in CATEGORICAL_COLUMNS:\n",
    "        return tf.string\n",
    "    else:\n",
    "        return tf.float32\n",
    "\n",
    "def serving_input_fn():\n",
    "    feature_placeholders = {\n",
    "        column: tf.placeholder(column_to_dtype(column), [None])\n",
    "        for column in FEATURE_COLUMNS\n",
    "    }\n",
    "    # DNNCombinedLinearClassifier expects rank 2 Tensors, but inputs should be\n",
    "    # rank 1, so that we can provide scalars to the server\n",
    "    features = {\n",
    "        key: tf.expand_dims(tensor, -1)\n",
    "        for key, tensor in feature_placeholders.items()\n",
    "    }\n",
    "    \n",
    "    return input_fn_utils.InputFnOps(\n",
    "        features, # input into graph\n",
    "        None,\n",
    "        feature_placeholders # tensor input converted from request \n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1373: get_timestamped_export_dir (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1378: get_temp_export_dir (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1388: get_input_alternatives (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1402: get_output_alternatives (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1412: build_all_signature_defs (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py:267: build_standardized_signature_def (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758070/model.ckpt-2002\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: models/model_WIDE_AND_DEEP_1571758070/export/temp-1571758355/saved_model.pb\n",
      "wide and deep model exported successfully to b'models/model_WIDE_AND_DEEP_1571758070/export/1571758355'\n"
     ]
    }
   ],
   "source": [
    "# 导出模型\n",
    "export_folder = m.export_savedmodel(\n",
    "    export_dir_base = model_dir + '/export',\n",
    "    serving_input_fn = serving_input_fn\n",
    ")\n",
    "\n",
    "print('wide and deep model exported successfully to {}'.format(export_folder))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "INFO:tensorflow:SavedModel written to: models/model_WIDE_AND_DEEP_1571737173/export/temp-1571756164/saved_model.pb  \n",
    "wide and deep model exported successfully to b'models/model_WIDE_AND_DEEP_1571737173/export/1571756164'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 七、其他\n",
    "\n",
    "    另一种训练模型的方法，是利用TensorFlow提供的“Experiments”框架。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.contrib.learn.python.learn import learn_runner\n",
    "from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils\n",
    "\n",
    "# 方法一：\n",
    "# 之前我们自己定义的函数：get_model(model_type = 'WIDE_AND_DEEP', model_dir=model_dir)\n",
    "\n",
    "#方法二：\n",
    "def experiment_fn(output_dir):\n",
    "    \n",
    "    print(output_dir)\n",
    "    \n",
    "    train_input_fn = generate_input_fn(train_file, BATCH_SIZE)\n",
    "    eval_input_fn = generate_input_fn(eval_file)\n",
    "    my_model = get_model(model_type=MODEL_TYPE, \n",
    "                  model_dir=output_dir)\n",
    "\n",
    "    experiment = tf.contrib.learn.Experiment(\n",
    "        my_model,\n",
    "        train_input_fn=train_input_fn,\n",
    "        eval_input_fn=eval_input_fn,\n",
    "        train_steps=1000\n",
    "        ,\n",
    "        export_strategies=[saved_model_export_utils.make_export_strategy(\n",
    "            serving_input_fn,\n",
    "            default_output_alternative_key=None,\n",
    "            exports_to_keep=1\n",
    "        )]\n",
    "    )\n",
    "    return experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/model_WIDE_AND_DEEP_1571758070\n",
      "Model directory = models/model_WIDE_AND_DEEP_1571758070\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x1367a9ba8>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_protocol': None, '_session_config': None, '_save_checkpoints_steps': 100, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_WIDE_AND_DEEP_1571758070'}\n",
      "estimator built\n",
      "WARNING:tensorflow:From <ipython-input-27-d10f15aefed3>:26: make_export_strategy (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py:484: ExportStrategy.__new__ (from tensorflow.contrib.learn.python.learn.export_strategy) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.estimator.train_and_evaluate, and use tf.estimator.Exporter.\n",
      "WARNING:tensorflow:From <ipython-input-27-d10f15aefed3>:26: Experiment.__init__ (from tensorflow.contrib.learn.python.learn.experiment) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.estimator.train_and_evaluate. You will also have to convert to a tf.estimator.Estimator.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/monitors.py:279: BaseMonitor.__init__ (from tensorflow.contrib.learn.python.learn.monitors) is deprecated and will be removed after 2016-12-05.\n",
      "Instructions for updating:\n",
      "Monitors are deprecated. Please use tf.train.SessionRunHook.\n",
      "INFO:tensorflow:Skipping training since max_steps has already saved.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:33:07Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758070/model.ckpt-2002\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:33:12\n",
      "INFO:tensorflow:Saving dict for global step 2002: accuracy = 0.763175, accuracy/baseline_label_mean = 0.252375, accuracy/threshold_0.500000_mean = 0.763175, auc = 0.7148175, auc_precision_recall = 0.4634353, global_step = 2002, labels/actual_label_mean = 0.252375, labels/prediction_mean = 0.25357863, loss = 0.50865465, precision/positive_threshold_0.500000_mean = 0.62263405, recall/positive_threshold_0.500000_mean = 0.15641406\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758070/model.ckpt-2002\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: models/model_WIDE_AND_DEEP_1571758070/export/Servo/temp-1571758392/saved_model.pb\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py:481: garbage_collect_exports (from tensorflow.contrib.learn.python.learn.utils.saved_model_export_utils) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Switch to tf.estimator.Exporter and associated utilities.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py:394: largest_export_versions (from tensorflow.contrib.learn.python.learn.utils.gc) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please implement your own file management or use Saver.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py:395: negation (from tensorflow.contrib.learn.python.learn.utils.gc) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please implement your own file management or use Saver.\n",
      "WARNING:tensorflow:From /Users/apple/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py:397: get_paths (from tensorflow.contrib.learn.python.learn.utils.gc) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please implement your own file name management.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.7 s, sys: 445 ms, total: 15.1 s\n",
      "Wall time: 13.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# 训练和校验模型\n",
    "exp = experiment_fn(model_dir)\n",
    "\n",
    "exp.train_and_evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CPU times: user 14.7 s, sys: 395 ms, total: 15.1 s  \n",
    "Wall time: 13.4 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-29-5739b5d44b2b>:2: run (from tensorflow.contrib.learn.python.learn.learn_runner) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.estimator.train_and_evaluate.\n",
      "models/model_WIDE_AND_DEEP_1571758410\n",
      "Model directory = models/model_WIDE_AND_DEEP_1571758410\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x144774710>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_protocol': None, '_session_config': None, '_save_checkpoints_steps': 100, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_WIDE_AND_DEEP_1571758410'}\n",
      "estimator built\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:33:51Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758410/model.ckpt-0\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:33:56\n",
      "INFO:tensorflow:Saving dict for global step 0: accuracy = 0.719825, accuracy/baseline_label_mean = 0.252375, accuracy/threshold_0.500000_mean = 0.719825, auc = 0.5623565, auc_precision_recall = 0.36067832, global_step = 0, labels/actual_label_mean = 0.252375, labels/prediction_mean = 0.13518457, loss = 65.92929, precision/positive_threshold_0.500000_mean = 0.39707515, recall/positive_threshold_0.500000_mean = 0.21248142\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Validation (step 1): loss = 65.92929, accuracy = 0.719825, labels/prediction_mean = 0.13518457, labels/actual_label_mean = 0.252375, accuracy/baseline_label_mean = 0.252375, auc = 0.5623565, auc_precision_recall = 0.36067832, accuracy/threshold_0.500000_mean = 0.719825, precision/positive_threshold_0.500000_mean = 0.39707515, recall/positive_threshold_0.500000_mean = 0.21248142, global_step = 0\n",
      "INFO:tensorflow:loss = 23.520302, step = 2\n",
      "INFO:tensorflow:Saving checkpoints for 102 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 7.56786\n",
      "INFO:tensorflow:loss = 0.45434478, step = 202 (4.873 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 204 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 61.0288\n",
      "INFO:tensorflow:Saving checkpoints for 306 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 59.3797\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:34:08Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758410/model.ckpt-306\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:34:12\n",
      "INFO:tensorflow:Saving dict for global step 306: accuracy = 0.75125, accuracy/baseline_label_mean = 0.252375, accuracy/threshold_0.500000_mean = 0.75125, auc = 0.6778915, auc_precision_recall = 0.4111198, global_step = 306, labels/actual_label_mean = 0.252375, labels/prediction_mean = 0.2389583, loss = 0.52920157, precision/positive_threshold_0.500000_mean = 0.61205566, recall/positive_threshold_0.500000_mean = 0.03922734\n",
      "INFO:tensorflow:Validation (step 325): loss = 0.52920157, accuracy = 0.75125, labels/prediction_mean = 0.2389583, labels/actual_label_mean = 0.252375, accuracy/baseline_label_mean = 0.252375, auc = 0.6778915, auc_precision_recall = 0.4111198, accuracy/threshold_0.500000_mean = 0.75125, precision/positive_threshold_0.500000_mean = 0.61205566, recall/positive_threshold_0.500000_mean = 0.03922734, global_step = 306\n",
      "INFO:tensorflow:loss = 0.53411764, step = 402 (11.262 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 408 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 10.7768\n",
      "INFO:tensorflow:Saving checkpoints for 510 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.4233\n",
      "INFO:tensorflow:loss = 0.489189, step = 602 (2.981 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 612 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.0243\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:34:22Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758410/model.ckpt-612\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:34:26\n",
      "INFO:tensorflow:Saving dict for global step 612: accuracy = 0.7585, accuracy/baseline_label_mean = 0.252375, accuracy/threshold_0.500000_mean = 0.7585, auc = 0.6905415, auc_precision_recall = 0.43567675, global_step = 612, labels/actual_label_mean = 0.252375, labels/prediction_mean = 0.2567736, loss = 0.52287203, precision/positive_threshold_0.500000_mean = 0.6422498, recall/positive_threshold_0.500000_mean = 0.097275876\n",
      "INFO:tensorflow:Validation (step 691): loss = 0.52287203, accuracy = 0.7585, labels/prediction_mean = 0.2567736, labels/actual_label_mean = 0.252375, accuracy/baseline_label_mean = 0.252375, auc = 0.6905415, auc_precision_recall = 0.43567675, accuracy/threshold_0.500000_mean = 0.7585, precision/positive_threshold_0.500000_mean = 0.6422498, recall/positive_threshold_0.500000_mean = 0.097275876, global_step = 612\n",
      "INFO:tensorflow:Saving checkpoints for 714 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 10.1636\n",
      "INFO:tensorflow:loss = 0.51375556, step = 802 (11.527 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 816 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.3635\n",
      "INFO:tensorflow:Saving checkpoints for 918 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 67.5749\n",
      "INFO:tensorflow:loss = 0.5642081, step = 1002 (3.135 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1002 into models/model_WIDE_AND_DEEP_1571758410/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.5642081.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-10-22T15:34:36Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758410/model.ckpt-1002\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2019-10-22-15:34:40\n",
      "INFO:tensorflow:Saving dict for global step 1002: accuracy = 0.7588, accuracy/baseline_label_mean = 0.252375, accuracy/threshold_0.500000_mean = 0.7588, auc = 0.7031491, auc_precision_recall = 0.44872665, global_step = 1002, labels/actual_label_mean = 0.252375, labels/prediction_mean = 0.28727108, loss = 0.51752025, precision/positive_threshold_0.500000_mean = 0.57057154, recall/positive_threshold_0.500000_mean = 0.1789995\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1571758410/model.ckpt-1002\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: models/model_WIDE_AND_DEEP_1571758410/export/Servo/temp-1571758480/saved_model.pb\n",
      "TensorFlow-Experiments wide and deep model Accuracy: 0.7588000297546387\n",
      "TensorFlow-Experiments wide and deep Model exported to [b'models/model_WIDE_AND_DEEP_1571758410/export/Servo/1571758480']\n"
     ]
    }
   ],
   "source": [
    "model_dir = create_model_dir(model_type = MODEL_TYPE)\n",
    "metrics, output_folder = learn_runner.run(experiment_fn, model_dir)\n",
    "\n",
    "print('TensorFlow-Experiments wide and deep model Accuracy: {}'.format(metrics['accuracy']))\n",
    "print('TensorFlow-Experiments wide and deep Model exported to {}'.format(output_folder))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TensorFlow-Experiments wide and deep model Accuracy: 0.7550749778747559  \n",
    "TensorFlow-Experiments wide and deep Model exported to [b'models/model_WIDE_AND_DEEP_1571757604/export/Servo/1571757690']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
